/**
 * 爬取热点仓库，保存到sqlite
 */
const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs')
const { db, saveMaven } = require('./db-helper')
const { httpGet } = require('./http-util')
const { log, wait } = require('./util')

const DOMAIN = 'https://mvnrepository.com'
const USAGES = 1000
const MAVEN_REPO_URL = 'https://repo1.maven.org/maven2'

/**
 * 保存各个分类的热点仓库到 group-domain 文件
 */
async function saveCategories() {
    const reptileUrl = `${DOMAIN}/open-source?p=`;
    let urls = []
    for (let i = 1; i <= 10; i++) {
        let res = await httpGet(reptileUrl + i)

        let $ = cheerio.load(res)
        $('#maincontent table a').each((i, elem) => {
            if ($(elem).html().indexOf('more') != -1) {
                urls.push(DOMAIN + $(elem).attr('href'))
            }
        })
    }
    log.debug('分类数：' + urls.length)
    saveGroupDomain(urls)
}

/**
 * 保存每个分类中 usages 超过1000的库
 * @param {*} url 
 */
async function getGroupId(categoryUrl) {
    let url = categoryUrl + '?p='
    let groupDomain = []
    for (let i = 1, usages = 99999; i <= 10 && usages > USAGES; i++) {
        await wait(1000);
        let res = await httpGet(url + i)
        let $ = cheerio.load(res)

        if (!$('.im-header').length) {
            break;
        }

        $('.im-header').each((i, elem) => {
            let $usage = $(elem).find('.im-usage b')
            usages = Number($usage.length && $usage.html().replace(',', ''))
            if (usages >= USAGES) {
                let groupId = $(elem).find('.im-subtitle a:eq(0)').html()
                log.debug(`groupId=${groupId}, usages=${usages}`)
                if (groupDomain.indexOf(groupId) == -1) {
                    groupDomain.push(groupId)
                }
            }
        })
    }
    return groupDomain;
}

/**
 * 保存group domain 到文件
 */
async function saveGroupDomain(categories) {

    for (let i in categories) {
        let url = categories[i]
        let group = await getGroupId(url)

        fs.appendFileSync('./group-domain', group.join('\n') + '\n')
    }
}

/**
 * 读取 group domain 文件，根据内容爬取maven目录
 */
async function loadGroupId() {
    let groupDomain = fs.readFileSync('./group-domain', { encoding: 'utf-8' }).split('\n')
    log.debug('group domain length : ' + groupDomain.length)

    groupDomain
        .map(line => [MAVEN_REPO_URL + '/' + line.replace(/\./g, '/') + '/'])
        .forEach(saveToMaven)
}

/**
 * 递归查询maven仓库
 * @param {*} url 
 */
async function saveToMaven(urls) {
    for (let k in urls) {
        let url = urls[k]
        let res = await httpGet(url)
        let $ = cheerio.load(res)

        let dirs = []
        $('pre a').each((i, elem) => {
            let $a = $(elem)
            if ($a.attr('href') != '../' && $a.attr('href').endsWith('/')) {
                dirs.push(url + $a.attr('href'))
            }
        })
        log.debug(dirs)
        if (dirs.length) {
            saveToMaven(dirs)
        } else {
            //出于性能考虑，如果已到达文件目录，保存该目录同级的数据，并退出函数
            //个别仓库会出现jar文件所在目录深度不同的情况，此时groupid、artifactId、version是错误的，这些数据在下载时再做进一步判断
            let mavens = urls.map(url => {
                let mavenPath = url.substring(MAVEN_REPO_URL.length).replace(/\//g, ' ').trim().split(' ')
                let version = mavenPath.pop()
                let artifactId = mavenPath.pop()
                let groupId = mavenPath.join('.')
                console.log(`groupid=${groupId}, artifactId=${artifactId}, version=${version}`)
                return { groupId, artifactId, version }
            })
            //保存到数据库
            await saveMaven(mavens)
            return
        }
    }
}

// 执行1：保存各个分类的热点仓库到 group-domain 文件，文件内保存仓库的 groupId 前缀
// saveCategories()
// 执行完成后可手动编辑 group-domain 文件，进行增、删、去重操作。

// 执行2：根据 group-domain 文件的内容，爬取maven仓库中 groupId 前缀开始的所有下级节点，并保存到sqlite
loadGroupId()
